#ifndef __PAGELIBPREPROCESSOR_H
#define __PAGELIBPREPROCESSOR_H
#include "Mylogger.hh"
#include "PagePreprocess/WebPage.h"
#include "PagePreprocess/tinyxml2.h"
#include "../lib/simhash/jenkins.h"
#include "../lib/simhash/Simhasher.hpp"
#include "../../lib/cppjieba/Jieba.hpp"
#include <string>
#include <iostream>
#include <vector>
#include <map>
#include <set>
#include <fstream>
#include <sstream>
#include <cstdlib>
using cppjieba::Jieba;
using simhash::Simhasher;
using std::cout;
using std::ifstream;
using std::istringstream;
using std::map;
using std::ostringstream;
using std::pair;
using std::set;
using std::string;
using std::unordered_map;
using std::vector;
using tinyxml2::XMLDocument;
using tinyxml2::XMLElement;

class PageLibPreprocessor
{
public:
    PageLibPreprocessor();
    void Preprocessor();
    void bulidInvertIndex();

private:
    void LodoffsetLib(const string &path, map<int, pair<int, int>> &off_set);           // 加载网页偏移库
    WebPage *LodPageLib(int id, const string &path, map<int, pair<int, int>> &off_set); // 加载指定id的网页库
    bool compare(WebPage *web);                                                         // 查重
    void store(WebPage *web);                                                           // 存放网页
    void store_newoffset();                                                             // 存放新的网页偏移库
private:
    Jieba _jieba;
    Simhasher _simhasher;
    XMLDocument _xmldoc;
    map<int, pair<int, int>> _offset;                        // 存放网页偏移库
    map<int, pair<int, int>> _newoffset;                     // 存放新的网页偏移库id,偏移量，长度
    map<int, int> _feature;                                  // 网页特征值库

    unordered_map<string, map<int, double>> _invertIndexLib; // 倒排索引库，单词，id，权重
    map<int, map<string, int>> _wordfrequency;               // 单词词频,id,单词，频率
    set<string> _stop_ch;                                    // 中文停止词集合
    map<int, double> _sum_wight;                             // 每个文章的权重和
};
#endif